import time
import pymongo
import pymysql
import requests
from lxml import etree

url = 'https://quotes.toscrape.com/page/1/'
res = requests.get(url)
data = res.content
tree = etree.HTML(data)
quotes = tree.xpath('//div[@class="quote"]')
# print(quotes)


# 使用mangodb存储
com = pymongo.MongoClient('localhost')
db = com.get_database('test2_db')
collection = db.get_collection('test_collection')
collection.delete_many({})
for item in quotes:
    # print(item)
    datas = item.xpath('./span[@class="text"]/text()')[0]
    author = item.xpath('.//small[@class="author"]/text()')[0]
    tags = item.xpath('.//div[@class="tags"]/a/text()')
    print(author)
    collection.insert_one({
        'quote': datas,
        'author': author,
        'tags': tags
    })
    time.sleep(1)
com.close()

# 使用sql语句进行存储
# data = []
# for item in quotes:
#     # print(item)
#     datas = item.xpath('./span[@class="text"]/text()')[0]
#     author = item.xpath('.//small[@class="author"]/text()')[0]
#     tags = item.xpath('.//div[@class="tags"]/a/text()')
#     print(author)
#     data.append({
#         "content": datas,
#         "author": author,
#         "tags": tags
#     })
#
#
# con = pymysql.Connect(user="root", password="123456")
# cur = con.cursor()
# cur.execute("create  database if not exists scrape charset=utf8")
# cur.execute("use scrape")
# cur.execute('create table if not exists items (id int not null primary key auto_increment, content varchar(500) not null, author varchar(50) not null)')
# cur.execute('create table if not exists tags (id int not null primary key auto_increment, title varchar(50) not null, items_id int not null, constraint fk_items_id foreign key (items_id) references items(id) on delete cascade on update cascade )')
# cur.execute('delete from tags')
# cur.execute('delete from items')
# con.commit()
# item_id = 100
# for d in data:
#     cur.execute('insert into items values (%s,%s,%s)', args=(item_id, d['content'], d['author']))
#     tags = [(0, tag, item_id) for tag in d['tags']]
#     cur.executemany('insert into tags values (%s,%s,%s)', args=tags)
#
#     item_id += 1
# con.commit()
# con.close()



